import re

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor

from scrapy.selector import HtmlXPathSelector

from hmss.items import IPItem 

RE_STYLE = r'\n*(?P<name>\.[-\w]+)\{(?P<val>.*?)\}'
RE_CLASS_ELEM = r'<(?:span|div) class="([-\w]+)">([\.\w]+)</(?:span|div)>'
RE_SUB_DISP_NONE = r'<(span|div) style="display:none".*?</(span|div)>'
RE_IP_FRAG = r'>([\d.]+)<'
RE_IP = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
RE_PORT = r'\d{2,4}'
RE_SPEED = r'width: (\d+).*'

MIN_ACCEPTED_SPEED = 70

class HideMyAssSpider(CrawlSpider):
    name = 'hidemyass'
    start_urls = [
    'http://proxylist.hidemyass.com/'
    ]
    allowed_domains = ['hidemyass.com']

    rules = (
        ## Linux From Scratch - Chapter 3.2 - All Packages
        Rule(SgmlLinkExtractor(restrict_xpaths=('//section')),callback='parseItem', follow=True),
        )

    def parseItem(self, response):
        #import pdb;pdb.set_trace()

        hxs = HtmlXPathSelector(response)
        links = hxs.select('//tr[@rel]')
        items = []
        for link in links:
            # skip other than fast ones
			speed_indicator = re.search(RE_SPEED, link.select('td[6]/div/div').extract()[0])
			if int(speed_indicator.group(1)) < MIN_ACCEPTED_SPEED:
				continue

            # skip other than higly anonymous
            if link.select('td[8]/text()').extract()[0].find('High') == -1:
                continue

            data = link.extract()
            styles = dict(re.findall(RE_STYLE, data))

            for class_elem in re.findall(RE_CLASS_ELEM, data):
                style = '.%s' % class_elem[0]
                if styles.has_key(style):
                    data = re.sub(r'class="%s"' % class_elem[0], 'style="%s"' % styles[style], data)

            data = re.sub(RE_SUB_DISP_NONE, '', data)

            ip_frag = re.findall(RE_IP_FRAG, data)
            ipaddress = ''.join(ip_frag)
            port = link.select('td[3]/text()').extract()[0].strip('\n')
            if re.match(RE_IP, ipaddress) and re.match(RE_PORT, port):
            	item = IPItem()
            	item['ipaddress'] = ipaddress
            	item['port'] = port
            	items.append(item)
        return items
